import re
from bs4 import BeautifulSoup
import urllib.request, urllib.error
import xlwt
import sqlite3

# 得到指定一个URL的网页内容
def askURL(url):
    head = {  # 模拟浏览器头部信息，向豆瓣服务器发送消息
        "User-Agent": "Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 80.0.3987.122  Safari / 537.36"
    }
    # 用户代理，表示告诉豆瓣服务器，我们是什么类型的机器、浏览器（本质上是告诉浏览器，我们可以接收什么水平的文件内容）

    request = urllib.request.Request(url, headers=head)
    html = ""
    try:
        response = urllib.request.urlopen(request)
        html = response.read().decode("utf-8")
    except urllib.error.URLError as e:
        if hasattr(e, "code"):
            print(e.code)
        if hasattr(e, "reason"):
            print(e.reason)
    return html

# 保存数据到表格
def saveData(datalist,savepath):
    print("save.......")
    book = xlwt.Workbook(encoding="utf-8",style_compression=0) #创建workbook对象
    sheet = book.add_sheet('bookmark', cell_overwrite_ok=True) #创建工作表
    col = ("title","url","description","tags")
    for i in range(0,4):
        sheet.write(0,i,col[i])  #列名
    for i in range(0,len(datalist)):
        # print("第%d条" %(i+1))       #输出语句，用来测试
        data = datalist[i]
        for j in range(0,4):
            sheet.write(i+1,j,data[j])  #数据
    book.save(savepath) #保存
html = askURL("https://www.tboxn.com/")


soup = BeautifulSoup(html, "html.parser")
posts_item = soup.find_all("article", class_ = re.compile("^posts-item sites-item"))
print(len(posts_item))
data = []
for item in posts_item:
    item_title = item.find("h3", class_ = re.compile("^item-title"))
    item_tags = item.find_all("a", class_="badge vc-l-theme text-ss mr-1")
    tags = []
    for tag in item_tags:
        tags.append(tag.text)
    item_detail_url = item.find("a", class_ = "sites-body")
    item_detail_desc = item_detail_url.find("div", class_ = re.compile("^line1 text-muted"))
    result = []
    result.append(item_title.text)
    result.append(item_detail_url.get("data-url"))
    result.append(item_detail_desc.text)
    result.append(",".join(tags))
    data.append(result)
saveData(data, "demo02.xls")
